import scrapy
from selenium import webdriver
from selenium.webdriver.edge.service import Service
from wangyiNewPro.items import WangyinewproItem
class WangyinewSpider(scrapy.Spider):
    name = "wangyiNew"
    # allowed_domains = ["www.xxx.com"]
    start_urls = ["https://news.163.com/"]
    models_urls = []
    # 实例化一个浏览器对象
    def __init__(self):
        service = Service()
        service.path = r'E:\work\python-crawler\第八章_scrapy框架\chromedriver.exe'
        # 启动浏览器
        self.browser = webdriver.Chrome(service=service)

    def parse(self, response):

        li_list = response.xpath('//*[@id="index2016_wrap"]/div[3]/div[2]/div[2]/div[2]/div/ul/li')

        a_index = [1,2,4,5]

        for index in a_index:
            model_url = li_list[index].xpath('./a/@href').extract_first()
            print(model_url)
            self.models_urls.append(model_url)

        # 依次对每一个模块对应的页面进行请求
        # 对每一个板块的URL进行请求发送
        for url in self.models_urls:
            yield scrapy.Request(url=url,callback=self.parse_model)

    #解析每一个板块中对应的标题和新闻详情页的URL
    def parse_model(self, response):
        div_list = response.xpath('/html/body/div/div[3]/div[3]/div[1]/div[1]/div/ul/li/div/div')
        for div in div_list:
            title = div.xpath('/html/body/div/div[3]/div[3]/div[1]/div[1]/div/ul/li/div/div[1]/div/div[1]/h3/a/text()').extract_first()
            detail_url = div.xpath('/html/body/div/div[3]/div[3]/div[1]/div[1]/div/ul/li/div/div[1]/a/@href').extract_first()
            item = WangyinewproItem()
            item['title'] = title
            # print(title,'        ',detail_url)
            # 对新闻详情页的URL发起请求
            yield scrapy.Request(url=detail_url,callback=self.parse_detail,meta={'item':item})

    def parse_detail(self, response): #解析新闻内容
        content = response.xpath('//*[@id="content"]/div[2]//text()').extract()
        content = ''.join(content)
        item = response.meta['item']
        item['content'] = content
        yield item

    def close(self, spider):
        spider.browser.close()